import asyncio
import functools
from pyppeteer import launch
from pyppeteer import launcher
import ssl
import random
import time
import datetime
import re
import subprocess
from write_baidu_file import baidu_write_xiaofei_info
from insert_sql import insert_etl_db
import timeout_decorator
import logging
import math
from common_dir.common_def import timeout_exec


ssl._create_default_https_context = ssl._create_unverified_context

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S')
yesterday = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y%m%d')
tmp_store_tuia_cost_info_dir = 'tuia_xiaofei_info_dir'


class pyppeteer_crawl_tuia:

    def __init__(self, name_password_dict):

        self.name_password_dict = name_password_dict
        self.chromium_dir = "/bin/chromium-browser"
        self.login_url = 'https://www.tuia.cn/public.html#/signin'
        self.name_selecotor = '#app > div > div > div > div > div > form > div.form-collection > div:nth-child(1) > input[type=text]'
        self.password_selector = '#app > div > div > div > div > div > form > div.form-collection > div:nth-child(2) > input[type=password]'
        self.button_selector = '#app > div > div > div > div > div > form > button'
        # 推广数据url"
        self.tuiguang_url = "https://www.tuia.cn/private.html#/addata"

    async def login(self, semaphore, name, password):
        async with semaphore:
            try:
                browser = await launch(headless=True, executablePath=self.chromium_dir, autoClose=True, ignoreDefaultArgs=['--enable-automation'])
                page = await browser.newPage()
                await page.evaluateOnNewDocument('() =>{ Object.defineProperties(navigator,'
                                                      '{ webdriver:{ get: () => false } }) }')
                await page.goto(self.login_url, {'timeout': 1000 * 44})
                await asyncio.sleep(round(random.uniform(0.71, 1.59), 1))
                await page.type(self.name_selecotor, name, {'delay': random.randint(10, 30)})
                await page.type(self.password_selector, password, {'delay': random.randint(10, 30)})
                await asyncio.sleep(round(random.uniform(0.31, 1.09), 1))
                await asyncio.wait(
                    [page.waitForNavigation({'timeout': 1000 * 30}), page.click(self.button_selector)])
                await asyncio.sleep(round(random.uniform(2.71, 3.59), 1))
                # print(page.url)
                await page.goto(self.tuiguang_url, {'timeout': 1000 * 44})
                await asyncio.sleep(round(random.uniform(2.71, 3.59), 1))
                # 昨天按钮
                click_handle = await page.xpath(
                    '//*[@id="app"]/div/div[2]/div[2]/div/div[1]/div/div[2]/div/div[2]')
                await click_handle[0].click()
                time.sleep(round(random.uniform(0.11, 0.91), 1))
                click_handle = await page.xpath(
                    '//*[@id="app"]/div/div[2]/div[2]/div/div[1]/div/div[2]/div/div[2]')
                await click_handle[0].click()
                time.sleep(round(random.uniform(1.11, 2.11), 1))
                # 判断昨天是否有数据
                if await page.xpath(
                    '//*[@id="app"]/div/div[2]/div[2]/div/div[2]/div[1]/div/div/div/div/div[1]/div[3]'):
                    data_all_html_elements = await page.xpath(
                        '//*[@id="app"]/div/div[2]/div[2]/div/div[2]/div[1]/div/div/div/div/div[1]/div[3]')
                    for item in data_all_html_elements:
                        you_wu_data = await (await item.getProperty('textContent')).jsonValue()
                        logging.info(you_wu_data+"\t"+name+"\t"+password)
                        if you_wu_data == "暂无数据":
                            baidu_write_xiaofei_info(yesterday+'\t'+'0'+'\t{"'+name+'":"'+password+'"}\t'+'toufang_tuia', 'toufang_tuia', '', './tuia_xiaofei_info_dir/').write_file()
                else:
                    # 总条数
                    count_all_html_elements = await page.xpath(
                        '//*[@id="app"]/div/div[2]/div[2]/div/div[2]/div[2]/span[2]')
                    for item in count_all_html_elements:
                        count = await (await item.getProperty('textContent')).jsonValue()
                        count_num = re.findall(r"\d+", count)[0].strip()
                    # 当前页面展示条数
                    count_all_html_elements = await page.xpath(
                        '//*[@id="app"]/div/div[2]/div[2]/div/div[2]/div[2]/span[1]')
                    for item in count_all_html_elements:
                        count = await (await item.getProperty('textContent')).jsonValue()
                        count_num_cur = re.findall(r"\d+", count)[0].strip()
                    if count_num_cur == count_num:
                        # 汇总金额
                        sum_jine = []
                        for i in range(1, int(count_num) + 1):
                            jine_all_html_elements = await page.xpath(
                                '//*[@id="app"]/div/div[2]/div[2]/div/div[2]/div[1]/div/div/div/div/div[1]/div[2]/table/tbody/tr[%i]/td[10]' % i)
                            for item in jine_all_html_elements:
                                jine = await (await item.getProperty('textContent')).jsonValue()
                                print(jine)
                                sum_jine.append(jine.strip().replace(',', ''))
                        print(name, '########', '汇总金额：', math.ceil(sum(list(map(float, sum_jine))) * 100))
                        jine_sum = math.ceil(sum(list(map(float, sum_jine))) * 100 / 1.1)
                        if jine_sum:
                            # insert_etl_db(str(jine_sum), 'toufang_tuia', 'xm').insert_mysql()
                            baidu_write_xiaofei_info(yesterday + '\t' + str(jine_sum) + '\t{"' + name + '":"' + password + '"}\t' + 'toufang_tuia',
                                'toufang_tuia', '', './tuia_xiaofei_info_dir/').write_file()
                        else:
                            logging.info(name+",总条数和当前展示条数不一致，待验证")
            except Exception as e:
                logging.info(e)
                print(datetime.datetime.now(), "!!!!!!!!tuia####\t\t", name, password)
            finally:
                # pass
                await browser.close()

    async def main(self):

        semaphore = asyncio.Semaphore(4)
        tasks = [
            self.login(semaphore, name, password) for name, password in self.name_password_dict.items()
        ]
        await asyncio.wait(tasks)

    @staticmethod
    def offical(name_password_dict):

        asyncio.get_event_loop().run_until_complete(pyppeteer_crawl_tuia(name_password_dict).main())

        time_num = 15
        while True:
            orgin_name_password_list = []
            for k, v in name_password_dict.items():
                orgin_name_password_list.append('{"' + k + '":"' + v + '"}')
            miss = list(set(orgin_name_password_list).difference(
                set(baidu_write_xiaofei_info('', 'toufang_tuia', '', './tuia_xiaofei_info_dir/').read_name_password())))
            if miss:
                print('after ', time_num - 15, ' : miss---\t', miss)
                miss_dict_all = {}
                for single_dict_str in miss:
                    miss_dict_all.update(eval(single_dict_str))
                time.sleep(time_num)
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                loop.run_until_complete(pyppeteer_crawl_tuia(miss_dict_all).main())
                if time_num == 60:
                    break
                time_num += 15
            else:
                ys_noys = 1
                sum_xiaofei = baidu_write_xiaofei_info('', 'toufang_tuia', ys_noys, './tuia_xiaofei_info_dir/').read_cost()
                print(sum_xiaofei)
                if sum_xiaofei:
                    insert_etl_db(str(sum_xiaofei), 'toufang_tuia', 'xm').insert_mysql()
                break


@timeout_exec
@timeout_decorator.timeout(360, use_signals=True)
def tuia():
    name_password_dict = {
                          'user@163.com': 'passwd',
                          'user2@163.com': 'passwd'
                          }
    subprocess.call("rm -f {}/{}_data".format(tmp_store_tuia_cost_info_dir,
                                              (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime(
                                                  '%Y-%m-%d') + '/toufang_tuia'), shell=True)
    pyppeteer_crawl_tuia.offical(name_password_dict)



if __name__ == '__main__':

    tuia()
